# Computations
import numpy as np
import pandas as pd
import scipy.stats as stats

# sklearn
from sklearn import metrics
from sklearn.model_selection import StratifiedShuffleSplit

# Pytorch
import torch
from torch.autograd import Variable
import torchvision.transforms as transforms

# Visualisation libraries

## Text
from colorama import Fore, Back, Style
from IPython.display import Image, display, Markdown, Latex, clear_output

## progressbar
import progressbar

## plotly
from plotly.offline import init_notebook_mode, iplot 
import plotly.graph_objs as go
import plotly.offline as py
from plotly.subplots import make_subplots
import plotly.express as px

## seaborn
import seaborn as sns

## matplotlib
import matplotlib.pyplot as plt
from matplotlib.patches import Ellipse, Polygon
from matplotlib.font_manager import FontProperties
import matplotlib.colors as mcolors
plt.style.use('seaborn-whitegrid')
plt.rcParams['axes.labelsize'] = 14
plt.rcParams['xtick.labelsize'] = 12
plt.rcParams['ytick.labelsize'] = 12
plt.rcParams['text.color'] = 'k'
%matplotlib inline

import warnings
warnings.filterwarnings("ignore")

Home / Business Analytics and Modeling

Telco Customer Churn

• Preprocessing
• Exploratory Data Analysis
• Modeling: PyTorch Multi-layer Perceptron (MLP) for Binary Classification
• Modeling: XGBoost Classifier

Telco Customer Churn

In this article, we analyze and predict customer churn for Telco Customer Churn data.

Dataset

• Customers who left within the last month – the column is called Churn
• Services that each customer has signed up for – phone, multiple lines, internet, online security, online backup, device * protection, tech support, and streaming TV and movies
• Customer account information – how long they’ve been a customer, contract, payment method, paperless billing, monthly charges, and total charges
• Demographic info about customers – gender, age range, and if they have partners and dependents

Columns	Description
customerID	Customer ID
gender	Whether the customer is a male or a female
SeniorCitizen	Whether the customer is a senior citizen or not (1, 0)
Partner	Whether the customer has a partner or not (Yes, No)
Dependents	Whether the customer has dependents or not (Yes, No)
tenure	Number of months the customer has stayed with the company
PhoneService	Whether the customer has a phone service or not (Yes, No)
MultipleLines	Whether the customer has multiple lines or not (Yes, No, No phone service)
InternetService	Customer’s internet service provider (DSL, Fiber optic, No)
OnlineSecurity	Whether the customer has online security or not (Yes, No, No internet service)
OnlineBackup	Whether the customer has an online backup or not (Yes, No, No internet service)
DeviceProtection	Whether the customer has device protection or not (Yes, No, No internet service)
TechSupport	Whether the customer has tech support or not (Yes, No, No internet service)
StreamingTV	Whether the customer has streaming TV or not (Yes, No, No internet service)
StreamingMovies	Whether the customer has streaming movies or not (Yes, No, No internet service)
Contract	The contract term of the customer (Month-to-month, One year, Two years)
PaperlessBilling	Whether the customer has paperless billing or not (Yes, No)
PaymentMethod	The customer’s payment method (Electronic check, Mailed check, Bank transfer (automatic), Credit card (automatic))
MonthlyCharges	The amount charged to the customer monthly
TotalCharges	The total amount charged to the customer
Churn	Whether the customer churned or not (Yes or No)

Path = 'telco-customer-churn/WA_Fn-UseC_-Telco-Customer-Churn.csv'
Data = pd.read_csv(Path.split(".")[0]+'_STD.csv')
df = Data.drop(columns = ['customer ID'])
Labels_dict = dict(zip([0, 1], ['No', 'Yes']))
Target = 'Churn'
display(Data.head(6).style.hide_index().set_precision(2))

customer ID	Gender	Senior Citizen	Partner	Dependents	Tenure	Phone Service	Multiple Lines	Internet Service	Online Security	Online Backup	Device Protection	Tech Support	Streaming TV	Streaming Movies	Contract	Paperless Billing	Monthly Charges	Total Charges	Churn	Bank transfer (automatic)	Credit card (automatic)	Electronic check	Mailed check
7590-VHVEG	-1.01	-0.44	1.03	-0.65	-1.28	-3.05	-0.85	-0.29	-0.10	1.18	-0.17	-0.10	-0.22	-0.23	-0.83	0.83	-1.16	-0.99	0	-0.53	-0.53	1.41	-0.54
5575-GNVDE	0.99	-0.44	-0.97	-0.65	0.07	0.33	-0.85	-0.29	1.32	-0.17	1.18	-0.10	-0.22	-0.23	0.37	-1.21	-0.26	-0.17	0	-0.53	-0.53	-0.71	1.84
3668-QPYBK	0.99	-0.44	-0.97	-0.65	-1.24	0.33	-0.85	-0.29	1.32	1.18	-0.17	-0.10	-0.22	-0.23	-0.83	0.83	-0.36	-0.96	1	-0.53	-0.53	-0.71	1.84
7795-CFOCW	0.99	-0.44	-0.97	-0.65	0.51	-3.05	-0.85	-0.29	1.32	-0.17	1.18	1.31	-0.22	-0.23	0.37	-1.21	-0.75	-0.20	0	1.89	-0.53	-0.71	-0.54
9237-HQITU	-1.01	-0.44	-0.97	-0.65	-1.24	0.33	-0.85	1.00	-0.10	-0.17	-0.17	-0.10	-0.22	-0.23	-0.83	0.83	0.20	-0.94	1	-0.53	-0.53	1.41	-0.54
9305-CDSKC	-1.01	-0.44	-0.97	-0.65	-0.99	0.33	1.17	1.00	-0.10	-0.17	1.18	-0.10	1.10	1.09	-0.83	0.83	1.16	-0.65	1	-0.53	-0.53	1.41	-0.54

Train and Test Sets

def DatasetTargetDist(Inp, Target, Labels_dict, PD):
    # Table
    Table = Inp[Target].value_counts().to_frame('Count').reset_index(drop = False).rename(columns = {'index':Target})
    Table[Target] = Table[Target].replace(Labels_dict)
    Table['Percentage'] = np.round(100*(Table['Count']/Table['Count'].sum()),2)
    fig = make_subplots(rows=1, cols=2, horizontal_spacing = 0.02, column_widths=PD['column_widths'],
                        specs=[[{"type": "table"},{"type": "pie"}]])

    # Right
    fig.add_trace(go.Pie(labels=Table[Target].values, values=Table['Count'].values,
                         pull=PD['pull'], textfont=dict(size= PD['textfont']),
                         marker=dict(colors = PD['PieColors'], line=dict(color='black', width=1))), row=1, col=2)
    fig.update_traces(hole=PD['hole'])
    fig.update_layout(height = PD['height'], legend=dict(orientation="v"), legend_title_text= PD['legend_title'])
    
    # Left
    T = Table.copy()
    T['Percentage'] = T['Percentage'].map(lambda x: '%%%.2f' % x)
    Temp = []
    for i in T.columns:
        Temp.append(T.loc[:,i].values)

    fig.add_trace(go.Table(header=dict(values = list(Table.columns), line_color='darkslategray',
                                       fill_color= PD['TableColors'][0], align=['center','center'],
                                       font=dict(color='white', size=12), height=25), columnwidth = PD['tablecolumnwidth'],
                           cells=dict(values=Temp, line_color='darkslategray',
                                      fill=dict(color= [PD['TableColors'][1], PD['TableColors'][1]]),
                                  align=['center', 'center'], font_size=12, height=20)), 1, 1)

    fig.update_layout(title={'text': '<b>' + Target + '<b>', 'x':PD['title_x'],
                                     'y':PD['title_y'], 'xanchor': 'center', 'yanchor': 'top'})
    fig.show()

Pull = [0 for x in range((len(Labels_dict)-1))]
Pull.append(.05)
PD = dict(PieColors = ['SeaGreen', 'Tomato'], TableColors = ['Indigo','GhostWhite'], hole = .4,
          column_widths=[0.6, 0.4],textfont = 14, height = 350, tablecolumnwidth = [0.1, 0.1, 0.1],
          pull = Pull, legend_title = Target, title_x = 0.5, title_y = 0.8)
del Pull
DatasetTargetDist(Data, Target, Labels_dict, PD)

StratifiedKFold is a variation of k-fold which returns stratified folds: each set contains approximately the same percentage of samples of each target class as the complete set.

X = df.drop(columns = Target).values
y = df[Target].values

Test_Size = 0.3
sss = StratifiedShuffleSplit(n_splits=1, test_size=Test_Size, random_state=42)
_ = sss.get_n_splits(X, y)

for train_index, test_index in sss.split(X, y):
    # X
    if isinstance(X, pd.DataFrame):
        X_train, X_test = X.loc[train_index], X.loc[test_index]
    else:
        X_train, X_test = X[train_index], X[test_index]
    # y    
    if isinstance(y, pd.Series):
        y_train, y_test = y[train_index], y[test_index]
    else:
        y_train, y_test = y[train_index], y[test_index]
del sss

def Train_Test_Dist(X_train, y_train, X_test, y_test, PD, Labels_dict = Labels_dict):
    def ToSeries(x):
        if not isinstance(x, pd.Series):
            Out = pd.Series(x)
        else:
            Out = x.copy()
        return Out
    
    fig = make_subplots(rows=1, cols=3, horizontal_spacing = 0.02, column_widths= PD['column_widths'],
                        specs=[[{"type": "table"},{'type':'domain'}, {'type':'domain'}]])
    # Right
    C = 2
    for y in [ToSeries(y_train).replace(Labels_dict), ToSeries(y_test).replace(Labels_dict)]:
        fig.add_trace(go.Pie(labels= list(Labels_dict.values()),
                             values= y.value_counts().values, pull=PD['pull'],
                             textfont=dict(size=PD['textfont']),
                             marker=dict(colors = PD['PieColors'],
                             line=dict(color='black', width=1))), row=1, col=C)
        fig.update_traces(hole=.5)
        fig.update_layout(legend=dict(orientation="v"), legend_title_text= PD['legend_title'])
        C+=1
    
    # Left
    
    # Table
    Table = pd.DataFrame(data={'Set':['X_train','X_test','y_train','y_test'],
                       'Shape':[X_train.shape, X_test.shape, y_train.shape, y_test.shape]}).astype(str)
    T = Table.copy()
    Temp = []
    for i in T.columns:
        Temp.append(T.loc[:,i].values)
        
    TableColors = PD['TableColors']
    fig.add_trace(go.Table(header=dict(values = list(Table.columns), line_color='darkslategray',
                                       fill_color= TableColors[0], align=['center','center'],
                                       font=dict(color='white', size=12), height=25), columnwidth = PD['tablecolumnwidth'],
                           cells=dict(values=Temp, line_color='darkslategray',
                                      fill=dict(color= [TableColors[1], TableColors[1]]),
                                  align=['center', 'center'], font_size=12, height=20)), 1, 1)
    fig.update_layout(title={'text': '<b>' + 'Dataset Distribution' + '<b>', 'x':PD['title_x'],
                                     'y':PD['title_y'], 'xanchor': 'center', 'yanchor': 'top'})
    if not PD['height'] == None:
        fig.update_layout(height = PD['height'])
    fig.show()
    
PD.update(dict(column_widths=[0.3, 0.3, 0.3], tablecolumnwidth = [0.2, 0.4], height = 350, legend_title = Target))

Train_Test_Dist(X_train, y_train, X_test, y_test, PD)

Modeling: PyTorch Multi-layer Perceptron (MLP) for Binary Classification

A multi-layer perceptron (MLP) is a class of feedforward artificial neural network (ANN). The algorithm at each iteration uses the Cross-Entropy Loss to measure the loss, and then the gradient and the model update is calculated. At the end of this iterative process, we would reach a better level of agreement between test and predicted sets since the error would be lower from that of the first step.

def TorchSets(Set):
    # Inut: Arrays
    # GPU Cuda
    if isinstance(Set, (pd.DataFrame, pd.Series)):
        Set = Set.values
    if torch.cuda.is_available():
        if Set.ndim==1:
            Out = Variable(torch.from_numpy(Set).type(torch.LongTensor).cuda())
        else:
            Out = Variable(torch.from_numpy(Set).cuda())
    # CPU
    else:
        if Set.ndim==1:
            Out = Variable(torch.from_numpy(Set).type(torch.LongTensor))
        else:
            Out = Variable(torch.from_numpy(Set))
    return Out

# Tensors
X_train_tensor = TorchSets(X_train)
y_train_tensor = TorchSets(y_train)
X_test_tensor = TorchSets(X_test)
y_test_tensor = TorchSets(y_test)
    
Batch_size = 100
iteration_number = int(3e4)

epochs_number = int(iteration_number / (len(X_train) / Batch_size))

# Pytorch train and test sets
Train_set = torch.utils.data.TensorDataset(X_train_tensor, y_train_tensor)
Test_set = torch.utils.data.TensorDataset(X_test_tensor, y_test_tensor)

# data loader
train_loader = torch.utils.data.DataLoader(Train_set, batch_size = Batch_size, shuffle = False)
test_loader = torch.utils.data.DataLoader(Train_set, batch_size = Batch_size, shuffle = False)

class MLP_Model(torch.nn.Module):
    '''
    A MLP model with two hidden layers
    '''
    def __init__(self, input_Size, hidden_Size, output_Size):
        super(MLP_Model, self).__init__()
        
        # Input Layer to the 1st Layer:
        self.fc1 = torch.nn.Linear(input_Size, hidden_Size) 
        torch.nn.init.kaiming_uniform_(self.fc1.weight, nonlinearity='relu')
        self.act1 = torch.nn.ReLU()
        
        # 1st Layer to 2nd Layer
        self.fc2 = torch.nn.Linear(hidden_Size, int(hidden_Size/4))
        torch.nn.init.kaiming_uniform_(self.fc2.weight, nonlinearity='relu')
        self.act2 = torch.nn.ReLU()
        
        # 2nd layer to Output Layer
        self.fc3 = torch.nn.Linear(int(hidden_Size/4), output_Size)
        torch.nn.init.kaiming_uniform_(self.fc3.weight)
        self.act3 = torch.nn.Sigmoid()
    
    def forward(self, x):
        # Input Layer to the 1st Layer:
        out = self.fc1(x)
        # Non-linearity 1
        out = self.act1(out)
        
        # 1st Layer to 2nd Layer
        out = self.fc2(out)
        out = self.act2(out)
               
        # 2nd layer to Output Layer
        out = self.fc3(out)
        out = self.act3(out)
        return out

def Plot_history(history, Table_Rows = 25, yLim = 2):
    fig = make_subplots(rows=1, cols=2, horizontal_spacing = 0.02, column_widths=[0.6, 0.4],
                        specs=[[{"type": "scatter"},{"type": "table"}]])
    # Left
    fig.add_trace(go.Scatter(x= history['Iteration'].values, y= history['Loss'].astype(float).values.round(4),
                             line=dict(color='OrangeRed', width= 1.5), name = 'Loss'), 1, 1)
    fig.add_trace(go.Scatter(x= history['Iteration'].values, y= history['Accuracy'].astype(float).values,
                             line=dict(color='MidnightBlue', width= 1.5),  name = 'Accuracy'), 1, 1)
    fig.update_layout(legend=dict(x=0, y=1.1, traceorder='reversed', font_size=12),
                  dragmode='select', plot_bgcolor= 'white', height=600, hovermode='closest',
                  legend_orientation='h')
    fig.update_xaxes(range=[history.Iteration.min(), history.Iteration.max()],
                     showgrid=True, gridwidth=1, gridcolor='Lightgray',
                     showline=True, linewidth=1, linecolor='Lightgray', mirror=True, row=1, col=1)
    fig.update_yaxes(range=[0, yLim], showgrid=True, gridwidth=1, gridcolor='Lightgray',
                     showline=True, linewidth=1, linecolor='Lightgray', mirror=True, row=1, col=1)
    # Right
    ind = np.linspace(0, history.index[-1], Table_Rows, endpoint = True).round(0).astype(int)
    T = history[history.index.isin(ind)]
    T[['Loss','Accuracy']] = T[['Loss','Accuracy']].applymap(lambda x: '%.4e' % x)
    Temp = []
    for i in T.columns:
        Temp.append(T.loc[:,i].values)
    fig.add_trace(go.Table(header=dict(values = list(history.columns), line_color='darkslategray',
                                       fill_color='Navy', align=['center','center'],
                                       font=dict(color='white', size=12), height=25), columnwidth = [0.4, 0.4, 0.4],
                           cells=dict(values=Temp, line_color='darkslategray',
                                      fill=dict(color=['Lavender', 'white', 'white']),
                                      align=['center', 'center'], font_size=12,height=20)), 1, 2)
    fig.show()

Fitting the model

input_Size, output_Size = X_train.shape[1], len(Labels_dict)
hidden_Size = 256

# model
model = MLP_Model(input_Size, hidden_Size, output_Size)

# GPU
if torch.cuda.is_available():
    model.cuda()

# Cross Entropy Loss 
criterion= torch.nn.CrossEntropyLoss()

# Optimizer 
optimizer = torch.optim.SGD(model.parameters(), lr= 1e-2, momentum=.9)

# Traning the Model
Count = 0
Loss_list = []
Iteration_list = []
Accuracy_list = []
MSE_list = []
MAE_list = []
Steps = 10

Progress_Bar = progressbar.ProgressBar(maxval= iteration_number + int(iteration_number/5),
                                       widgets=[progressbar.Bar('=', '|', '|'),
                                                progressbar.Percentage()])
    
# print('---------------------------------------------------------')
for epoch in range(epochs_number):
    for i, (Xtr, ytr) in enumerate(train_loader):
        
        # Variables
        Xtr = Variable(Xtr.view(-1, input_Size))
        ytr = Variable(ytr)
        
        # Set all gradients to zero
        optimizer.zero_grad()
        
        # Forward
        Out = model(Xtr.float())
        
        # loss
        loss = criterion(Out, ytr.long())
        
        # Backward (Calculating the gradients)
        loss.backward()
        
        # Update parameters
        optimizer.step()
        
        Count += 1
        
        del Xtr, ytr
        
        # Predictions
        if Count % Steps == 0:
            # Calculate Accuracy         
            Correct, Total = 0, 0
            # Predictions
            for Xts, yts in test_loader: 
                Xts = Variable(Xts.view(-1, input_Size))
                
                # Forward
                Out = model(Xts.float())
                
                # The maximum value of Out
                Predicted = torch.max(Out.data, 1)[1]
                
                # Total number of yts
                Total += len(yts)
                
                # Total Correct predictions
                Correct += (Predicted == yts).sum()
            del Xts, yts
            # storing loss and iteration
            Loss_list.append(loss.data)
            Iteration_list.append(Count)
            Accuracy_list.append(Correct / float(Total))
            
        Progress_Bar.update(Count)

Progress_Bar.finish()

history = pd.DataFrame({'Iteration': np.array(Iteration_list),
                      'Loss': np.array([x.cpu().data.numpy() for x in Loss_list]),
                      'Accuracy': np.array([x.cpu().data.numpy() for x in Accuracy_list])})
del Loss_list, Iteration_list, Accuracy_list

|=========================================================================|100%

Model Performance

Plot_history(history, Table_Rows = 30, yLim = 1)

Confusion Matrix

The confusion matrix allows for visualization of the performance of an algorithm. Note that due to the size of data, here we don't provide a Cross-validation evaluation. In general, this type of evaluation is preferred.

def Confusion_Mat(CM_Train, CM_Test, PD, n_splits = 10):
    if n_splits == None:
        Titles = ['Train Set', 'Test Set']
    else:
        Titles = ['Train Set (CV = % i)' % n_splits, 'Test Set (CV = % i)' % n_splits]
    CM = [CM_Train, CM_Test]
    Cmap = ['Greens', 'YlGn','Blues', 'PuBu']
    for i in range(2):
        fig, ax = plt.subplots(1, 2, figsize= PD['FS'])
        fig.suptitle(Titles[i], weight = 'bold', fontsize = 16)
        _ = sns.heatmap(CM[i], annot=True, annot_kws={"size": PD['annot_kws']}, cmap=Cmap[2*i], ax = ax[0],
                        linewidths = 0.2, cbar_kws={"shrink": PD['shrink']})
        _ = ax[0].set_title('Confusion Matrix');
        Temp = np.round(CM[i].astype('float') / CM[i].sum(axis=1)[:, np.newaxis], 2)
        _ = sns.heatmap(Temp,
                        annot=True, annot_kws={"size": PD['annot_kws']}, cmap=Cmap[2*i+1], ax = ax[1],
                       linewidths = 0.4, vmin=0, vmax=1, cbar_kws={"shrink": PD['shrink']})
        _ = ax[1].set_title('Normalized Confusion Matrix');

        for a in ax:
            _ = a.set_xlabel('Predicted labels')
            _ = a.set_ylabel('True labels'); 
            _ = a.xaxis.set_ticklabels(PD['Labels'])
            _ = a.yaxis.set_ticklabels(PD['Labels'])
            _ = a.set_aspect(1)

# Train
y_pred = model(X_train_tensor.float())
y_pred = torch.max(y_pred.data, 1)[1]
y_pred = y_pred.cpu().data.numpy()
Reports_Train = pd.DataFrame(metrics.classification_report(y_train, y_pred, target_names=list(Labels_dict.values()),
                                                           output_dict=True)).T
CM_Train = metrics.confusion_matrix(y_train, y_pred)
# Test
y_pred = model(X_test_tensor.float())
y_pred = torch.max(y_pred.data, 1)[1]
y_pred = y_pred.cpu().data.numpy()
Reports_Test = pd.DataFrame(metrics.classification_report(y_test, y_pred, target_names=list(Labels_dict.values()),
                                                          output_dict=True)).T
CM_Test = metrics.confusion_matrix(y_test, y_pred)

Reports_Train = Reports_Train.reset_index().rename(columns ={'index': 'Train Set'})
Reports_Test = Reports_Test.reset_index().rename(columns ={'index': 'Test Set'})
                                                 
display(Reports_Train.style.hide_index().set_properties(**{'background-color': 'HoneyDew', 'color': 'Black'}).\
        set_properties(subset=['Train Set'], **{'background-color': 'SeaGreen', 'color': 'White'}))
display(Reports_Test.style.hide_index().set_properties(**{'background-color': 'Azure', 'color': 'Black'}).\
        set_properties(subset=['Test Set'], **{'background-color': 'RoyalBlue', 'color': 'White'}))

PD = dict(FS = (10, 6), annot_kws = 14, shrink = .6, Labels = list(Labels_dict.values()))
Confusion_Mat(CM_Train, CM_Test, PD = PD, n_splits = None)

Train Set	precision	recall	f1-score	support
No	0.908765	0.910271	0.909517	3622.000000
Yes	0.750384	0.746942	0.748659	1308.000000
accuracy	0.866937	0.866937	0.866937	0.866937
macro avg	0.829575	0.828606	0.829088	4930.000000
weighted avg	0.866744	0.866937	0.866839	4930.000000

Test Set	precision	recall	f1-score	support
No	0.854497	0.832474	0.843342	1552.000000
Yes	0.567388	0.607843	0.586919	561.000000
accuracy	0.772835	0.772835	0.772835	0.772835
macro avg	0.710943	0.720159	0.715131	2113.000000
weighted avg	0.778270	0.772835	0.775262	2113.000000

---

References

1. UC Irvine Machine Learning Repository: Breast Cancer Wisconsin (Diagnostic) Data Set
2. Artificial neural network. Retrieved June 02, 2020, from https://en.wikipedia.org/wiki/Artificial_neural_network.
3. PyTorch Documentation
4. Mower, Jeffrey P. "PREP-Mt: predictive RNA editor for plant mitochondrial genes." BMC bioinformatics 6.1 (2005): 1-15.

---